#!/usr/bin/env python
#:usage: python lazed.py main /opt/logfile/chn/main/ftptemp
import os,sys,re
import fnmatch
import time

#===globe counters===
logstatic={"readlines":0,"droplines":0,"all_clients":0,"all_hosts":0,"time":time.time(),"all_req":0.0,"all_sc":0.0,"all_cs":0.0}

g_clients={}
#"client-ip":[req_counts,denied_req_counts,scbytes,csbytes,{"denihost-ip":[req_counts,0,0,0]}
g_hosts={}
#"host-ip":[req_counts,denied_req_counts,scbytes,csbytes,{}]

g_days={}
# "2011-09-08":[req_counts,p,scbytes,csbytes]

all_log_types={"main":{"client-ip":3,"host-ip":15,"sc":22,"cs":23}, # ref ash810 main
              "cifs":{"client-ip":2,"host-ip":4,"sc":24,"cs":23,"grepword":"SUCCESS"},
              "ssl":{"client-ip":3,"host-ip":14,"sc":19,"cs":20}
             }

#output_fmt="{ip}|{req_count}|{denied_req}|{sc}|{cs}"
output_fmt="{0}|{1}|{2}|{3:0.4f}|{4:0.4f}"

def dic_topten(dic,f):
	top_s=sorted(dic.iteritems(),key=lambda x:x[1][f],reverse=True)
	#o_top_s=[]
	if len(top_s)>=10:
		#for i in top_s[0:10]:
		#	if (i[0] is not "--Empty--" ) and i[1][f] == 0: o_top_s.append(("--Empty--",[0,0,0,0]))
		#	else: o_top_s.append(i)
		return top_s[0:10]

	while len(top_s) < 10:
			top_s.append(("--Empty--",[0,0,0,0]))
	return top_s


def out_put(tops,headstring):
	#print headstring
	for i in tops:
		output_fmt="{0}|{1}|{req_per:0.2f}%|{2}|{3:0.4f}|{sc_per:0.2f}%|{4:0.4f}|{cs_per:0.2f}%"
		print output_fmt.format(i[0],i[1][0],i[1][1],i[1][2]/1073741824.00,i[1][3]/1073741824.00,
				req_per=i[1][0]*100.0/logstatic["all_req"],sc_per=i[1][2]*100.0/logstatic["all_sc"],
				cs_per=i[1][3]*100.0/logstatic["all_cs"])
def list_update(alist,r,dr,sc,cs):
	alist[0]+=r
	alist[1]+=dr
	alist[2]+=sc
	alist[3]+=cs

def dic_update(dic,key,sc=0,cs=0,dst=None): #
	if dst is not None: #update a denied record
		if key not in dic: dic[key]=[1,1,sc,cs,{dst:[1,0,0,0]}]
		elif dst not in dic[key][4]:
			list_update(dic[key],1,1,sc,cs)
			dic[key][4][dst]=[1,1,0,0]
		else:
			list_update(dic[key],1,0,sc,cs)
			list_update(dic[key][4][dst],1,0,0,0)
	else:
		if key not in dic: dic[key]=[1,0,sc,cs,{}]
		else: list_update(dic[key],1,0,sc,cs)

def killspacebyline(line):
	reg = re.compile(r'".*?"')
        for qued in re.findall(reg,line):
        	r=qued.replace(" ","_")
        	line=line.replace(qued,r)
        return line

def filterline(line,log_type):
	if line[0] is  "#":
		logstatic["droplines"]+=1
		return -1
	if "grepword" in log_type:
		if log_type["grepword"] not in line:
			logstatic["droplines"]+=1
			return -1

	k=killspacebyline(line).split()
	if "-" == k[log_type["sc"]] or "-" == k[log_type["cs"]]:
		logstatic["droplines"]+=1
		return -1

	if "-" == k[log_type["host-ip"]] or "-" == k[log_type["client-ip"]]:
		logstatic["droplines"]+=1
		return -1
	if "policy_denied" in line: return (1,k) #with dst
	else: return (0,k) #without dst

def coroute(func):
            def wrapper(*args,**kw):
                gen = func(*args, **kw)
                gen.next()
                return gen
            wrapper.__name__ = func.__name__
            wrapper.__dict__ = func.__dict__
            wrapper.__doc__  = func.__doc__
            return wrapper
@coroute
def find_files(tg):
	while True:
		topdir,pattern = (yield)
		for path,dirname,filelist in os.walk(topdir):
			for name in filelist:
				if fnmatch.fnmatch(name,pattern):
					print "#Reading file:",os.path.join(path,name)
					tg.send(os.path.join(path,name))

import gzip

@coroute
def opener(tg):
	while True:
		name = (yield)
		if name.endswith(".gz"): f = gzip.open(name)
		elif name.endswith(".done"): f = gzip.open(name)
		else: f = open(name)
		tg.send(f)

@coroute
def catlines(tg):
	while True:
		f = (yield)
		for line in f:
			tg.send(line)

@coroute
def procline(logtype):
	while True:
		line  = (yield)
		logstatic["readlines"]+=1
		f_res=filterline(line,logtype)
		if type(f_res) is tuple:
			fds=f_res[1]
                        dic_update(g_hosts,fds[log_type["host-ip"]],int(fds[log_type["sc"]]),
						int(fds[log_type["cs"]]))
                        dic_update(g_days,fds[0],int(fds[log_type["sc"]]),
						int(fds[log_type["cs"]]))
			if f_res[0] == 1: #dic_update(dic,key,sc=0,cs=0,dst=not_None)
				dic_update(g_clients,fds[log_type["client-ip"]],
							int(fds[log_type["sc"]]),
							int(fds[log_type["cs"]]),fds[log_type["host-ip"]])
			elif f_res[0] == 0:#dst=None
                               	dic_update(g_clients,fds[log_type["client-ip"]],
							int(fds[log_type["sc"]]),
							int(fds[log_type["cs"]]))
			else: pass


log_type=all_log_types[sys.argv[1]]
pyzcater = find_files(opener(catlines(procline(log_type))))
pyzcater.send((sys.argv[2],"*gz*"))

logstatic["all_clients"]=len(g_clients)
logstatic["all_hosts"]=len(g_hosts)
logstatic["time"]=time.time()-logstatic["time"]


#############Out put#################

print "#Report for ",sys.argv[1]
##static by days
print "#print 5 blanks"
print "###"
print "###"
print "###"
print "###"
print "###"
if len(g_days)==0: sys.exit(0)
k_of_days=sorted(g_days)
for i in k_of_days:
	#output_fmt="{0}|{1}|-|{2}|{3:0.4f}|-|{4:0.4f}|-"
	#print output_fmt.format(i,g_days[i][0],g_days[i][1],g_days[i][2]/1073741824.00,g_days[i][3]/1073741824.00)
	logstatic["all_req"]+=g_days[i][0]
	logstatic["all_sc"]+=g_days[i][2]
	logstatic["all_cs"]+=g_days[i][3]


##top hosts
print "#Top Hosts by Requests"
top_ips= dic_topten(g_hosts,0)
out_put(top_ips,"#Hosts "+"Requests denied SC-bytes CS-bytes")

print "#Top Hosts by SC-bytes"
top_ips= dic_topten(g_hosts,2)
out_put(top_ips,"#Hosts "+"Requests denied SC-bytes CS-bytes")

del g_hosts

##top Clients
print "#Top Clients by Requests"
top_ips= dic_topten(g_clients,0)
out_put(top_ips,"#Cliets "+"Requests denied SC-bytes CS-bytes")

print "#Top Clients by SC-bytes"
top_ips= dic_topten(g_clients,2)
out_put(top_ips,"#Cliets "+"Requests denied SC-bytes CS-bytes")

if sys.argv[1] not in "cifs":
	print "#Top Clients by Denied Requests"
	top_ips= dic_topten(g_clients,1)
	out_put(top_ips,"#Cliets "+"Requests denied SC-bytes CS-bytes")
	del g_clients

	print "#Details for: Top Clients by Denied Requests"
	for c,k in top_ips:
		print "["+c+"]"+"Most visited:"
		pblanks=0
		if "Empty" in c:
			pblanks=10
			while pblanks >0:
				pblanks -=1
				print "--Empty--"
		else:
			top_acc=dic_topten(k[4],0)
			out_put(top_acc,"#Hosts "+"Requests denied SC-bytes CS-bytes")

print "#readlines","all_clients","all_hosts"
logstatic_fmt="{0}|{1}|{2}"
print logstatic_fmt.format(logstatic["readlines"],logstatic["all_clients"],logstatic["all_hosts"])
#{"readlines":0,"droplines":0,"all_clients":0,"all_hosts":0,"time":time.time(),"all_req":0,"all_sc":0,"all_cs":0}
print logstatic

print "#Days..."
for i in k_of_days:
	output_fmt="{0}|{1}|-|{2}|{3:0.4f}|-|{4:0.4f}|-"
        print output_fmt.format(i,g_days[i][0],g_days[i][1],g_days[i][2]/1073741824.00,g_days[i][3]/1073741824.00)
pyzcater.close()
